﻿namespace DCHealthCheck {

    #region Usings
    using Amib.Threading;
    using Contoso.ActiveDirectory;
    using DCHealthCheck.Common;
    using DCHealthCheck.DomainModel;
    using Microsoft.GroupPolicy;
    using System;
    using System.Collections.Generic;
    using System.Configuration;
    using System.Data;
    using System.Data.Entity.Infrastructure;
    using System.Diagnostics;
    using System.IO;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;
    using System.Xml.Serialization; 
    #endregion

    public class HealthCheckRunner {

        #region Members

        /// <summary>
        /// Key: Site Name  Value: ADDataAccess.SiteProperties object
        /// </summary>
        private static Dictionary<string, SiteProperties> ADSites;

        private static int CDriveFreeSpaceMBWarning;

        private static int CDriveFreeSpaceMBError;

        /// <summary>
        /// Key: Tag object DN  Value: Most recent value found
        /// </summary>
        private static Dictionary<string, DateTime> CurrentTagObjectInfo;

        private static int DatabasePurgeIntervalDays;

        private static List<ComputerProperties> DCComputerAccounts;

        private static List<SystemInfoBuilder> DCsForTest;

        /// <summary>
        /// The relative distinguished name of the DC health check 'tag' objects
        /// </summary>
        private static string DCHealthCheckTagRDN;

        /// <summary>
        /// To limit the scope of the tests to a subset of DC's
        /// </summary>
        private static string DCNameFilter { get; set; }

        /// <summary>
        /// Key: Domain DNS name Value: GpoCollection
        /// </summary>
        private static Dictionary<string, List<Gpo>> DomainGPOs;

        /// <summary>
        /// A list of the PDC sysvol GPO folders.  Key: Domain DNS name.  Value: most recent folder located under sysvol\domain\Policies
        /// </summary>
        private static Dictionary<string, GroupPolicySysvolInfo> DomainPDCSysvolPolicyFolders;

        private static DateTime HealthCheckStartTime;

        private static StringBuilder HealthCheckSummary;

        private static TimeSpan HealthCheckTimeLimit;

        private static string HealthCheckUrl { get; set; }

        /// <summary>
        /// Key: Process executable file name with extension Value: Description
        /// </summary>
        private static Dictionary<string, string> HighCpuProcessesToSuspend { get; set; }

        /// <summary>
        /// Key: Domain DNS Name Value: Dictionary of FSMO Role Names and Owner DN's
        /// </summary>
        public static Dictionary<string, Dictionary<string, string>> FSMORoleOwners { get; private set; }

        private static log4net.ILog Log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);

        private static readonly DateTime SqlDateTimeMinValue = new DateTime(1753, 1, 1, 0, 0, 0, DateTimeKind.Utc);

        private static SmartThreadPool STP;

        /// <summary>
        /// Key: Process executable file name  Value: Process description
        /// </summary>
        private static Dictionary<string, string> RequiredProcesses;

        private static TimeSpan MaxProcessCPUTimePerDay;

        private static int MaxNtdsDitSizeMBThreshold;

        private static int MaxPendingReplicationsCountThreshold;

        private static int MaxTotalHandleCountThreshold;

        private static int MaxTotalThreadCountThreshold;

        private static int MaxUptimeDays;

        private static int PingLossPercentThreshold;

        private static string PSSuspendPath { get; set; }

        private static StringBuilder ServersNotTestedSummary;

        private static TimeSpan ServerAgeMinimumThreshold = new TimeSpan(2, 0, 0, 0, 0);

        private static TimeSpan SiteServerAgeMinimumThreshold = new TimeSpan(2, 0, 0, 0, 0);

        private static TimeSpan SiteServerNTDSSettingsAgeMinimumThreshold = new TimeSpan(2, 0, 0, 0, 0);

        #region Report Paths
        private static string ReportFolder { get; set; }

        private static string ReportPath { get; set; }

        private static string ReportSharePath { get; set; }

        private static string ReportSummaryPath { get; set; }
        #endregion

        #endregion

        #region Constructor
        static HealthCheckRunner() {

            HealthCheckStartTime = DateTime.UtcNow;
            int threadPoolConcurrency = 5 * Environment.ProcessorCount;
            if (threadPoolConcurrency > 50) threadPoolConcurrency = 50;
            STPStartInfo stpStartInfo = new STPStartInfo() { IdleTimeout = 20 * 1000, MaxWorkerThreads = threadPoolConcurrency };
            stpStartInfo.PostExecuteWorkItemCallback = STPPostExecute;
            STP = new SmartThreadPool(stpStartInfo);
            STP.Concurrency = threadPoolConcurrency;

            Log.InfoFormat("[ThreadID: {0}] {1} ThreadPool Concurrency: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), STP.Concurrency);

            DCsForTest = new List<SystemInfoBuilder>();
            ADSites = new Dictionary<string, SiteProperties>(StringComparer.OrdinalIgnoreCase);
            DCComputerAccounts = new List<ComputerProperties>();
            DomainPDCSysvolPolicyFolders = new Dictionary<string, GroupPolicySysvolInfo>(StringComparer.OrdinalIgnoreCase);
            DomainGPOs = new Dictionary<string, List<Gpo>>(StringComparer.OrdinalIgnoreCase);
            FSMORoleOwners = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
            HighCpuProcessesToSuspend = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

            CurrentTagObjectInfo = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
            RequiredProcesses = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

            HealthCheckSummary = new StringBuilder();
            ServersNotTestedSummary = new StringBuilder();

            GetAppSettings();

            ReportFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Reports");
            if (!Directory.Exists(ReportFolder)) Directory.CreateDirectory(ReportFolder);
            ReportPath = Path.Combine(ReportFolder, "DCHealthCheck_" + HealthCheckStartTime.ToString("yyyy-MMM-dd-HHmm") + ".csv");
            ReportSummaryPath = Path.Combine(ReportFolder, "DCHealthCheckSummary_" + HealthCheckStartTime.ToString("yyyy-MMM-dd-HHmm") + ".txt");
        } 
        #endregion

        #region Methods
        public static void DoWork() {
            Log.InfoFormat("[ThreadID: {0}] {1}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

            if ((Environment.OSVersion.Version.Major < 6) || (Environment.OSVersion.Version.Minor < 1)) {
                Log.FatalFormat("[ThreadID: {0}] {1} DCHealthCheck must be run on Windows Server 2008 R2 or higher.", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());
                return;
            }

            try {
                TestDatabaseAccess();
                Utility.FlushDNSCache();
                ADDataAccess.Initialize();

                FSMORoleOwners = StartupDataGenerator.GetFSMORoleOwners();
                DomainGPOs = StartupDataGenerator.GetDomainGPOs();
                DCComputerAccounts = StartupDataGenerator.GetDCComputerAccounts();

                Log.InfoFormat("[ThreadID: {0}] {1} Domain controller computer accounts found for testing: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DCComputerAccounts.Count);

                List<TagObject> tagObjects = StartupDataGenerator.GetTagObjects(DCHealthCheckTagRDN);
                DomainPDCSysvolPolicyFolders = StartupDataGenerator.GetPDCSysvolTestFolders();
                ADSites = StartupDataGenerator.GetADSites();

                foreach (var dcComputerAccount in DCComputerAccounts) {
                    if (!string.IsNullOrWhiteSpace(DCNameFilter)) {
                        if (!Regex.IsMatch(dcComputerAccount.DnsHostName, DCNameFilter, RegexOptions.IgnoreCase | RegexOptions.Compiled)) continue;
                    }

                    #region Get AD Site and SiteServer objects
                    SiteProperties adSite = null;
                    SiteServer siteServer = null;
                    if (ADSites.Values.SelectMany(x => x.SiteServers).Any(y => string.Equals(y.DnsHostName, dcComputerAccount.DnsHostName, StringComparison.OrdinalIgnoreCase))) {
                        adSite = ADSites
                        .Where(x => 
                            x.Value.SiteServers
                            .Any(y => string.Equals(y.DnsHostName, dcComputerAccount.DnsHostName, StringComparison.OrdinalIgnoreCase)))
                        .FirstOrDefault().Value;
                    }

                    if (adSite != null) {
                        siteServer = adSite.SiteServers
                            .Where(x => string.Equals(x.DnsHostName, dcComputerAccount.DnsHostName, StringComparison.OrdinalIgnoreCase))
                            .FirstOrDefault();
                    }
                    #endregion

                    #region Exclude servers that do not have a corresponding SiteServer object or SiteServer.NTDSSettings object
                    if (siteServer == null) {
                        Log.WarnFormat("[ThreadID: {0}] {1} Not testing server: {2} due to unable to locate SiteServer object.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount);

                        ServersNotTestedSummary.AppendLine(string.Format("DC Computer account not tested due to no corresponding SiteServer object: {0}", dcComputerAccount.DN));

                        continue;
                    }

                    if (siteServer.NTDSSettings == null) {
                        Log.WarnFormat("[ThreadID: {0}] {1} Not testing server: {2} due to unable to it does not have an NTDSSettings object.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount);

                        ServersNotTestedSummary.AppendLine(string.Format("SiteServer not tested due to no NTDSSettings object: {0}", siteServer.DN));

                        continue;
                    }
                    #endregion

                    #region Exclude servers that are recently created. They may be missing some info.
                    if (DateTime.UtcNow.Subtract(dcComputerAccount.WhenCreated) < ServerAgeMinimumThreshold) {
                        Log.InfoFormat("[ThreadID: {0}] {1} Not testing server: {2} due to computer account was recently created on: {3}.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount.DN, dcComputerAccount.WhenCreated.ToString("yyyy-MMM-dd HH:mm"));

                        ServersNotTestedSummary.AppendLine(string.Format("Not testing server: {0} due to computer account was recently created on: {1}", dcComputerAccount.DN, dcComputerAccount.WhenCreated.ToString("yyyy-MMM-dd HH:mm")));

                        continue;
                    }

                    if (DateTime.UtcNow.Subtract(siteServer.WhenCreated) < SiteServerAgeMinimumThreshold) {
                        Log.InfoFormat("[ThreadID: {0}] {1} Not testing server: {2} due to SiteServer object was recently created on: {3}.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount.DN, siteServer.WhenCreated.ToString("yyyy-MMM-dd HH:mm"));

                        ServersNotTestedSummary.AppendLine(string.Format("Not testing server: {0} due to SiteServer object was recently created on: {1}", dcComputerAccount.DN, siteServer.WhenCreated.ToString("yyyy-MMM-dd HH:mm")));

                        continue;
                    }

                    if (DateTime.UtcNow.Subtract(siteServer.NTDSSettings.WhenCreated) < SiteServerNTDSSettingsAgeMinimumThreshold) {
                        Log.InfoFormat("[ThreadID: {0}] {1} Not testing server: {2} due to SiteServer.NTDSSettings object was recently created on: {3}.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount.DN, siteServer.NTDSSettings.WhenCreated.ToString("yyyy-MMM-dd HH:mm"));

                        ServersNotTestedSummary.AppendLine(string.Format("Not testing server: {0} due to SiteServer.NTDSSettings object was recently created on: {1}", dcComputerAccount.DN, siteServer.NTDSSettings.WhenCreated.ToString("yyyy-MMM-dd HH:mm")));

                        continue;
                    }
                    #endregion

                    GroupPolicySysvolInfo pdcGroupPolicySysvolInfo = null;
                    if (!DomainPDCSysvolPolicyFolders.TryGetValue(dcComputerAccount.DNSDomain, out pdcGroupPolicySysvolInfo)) {
                        //continue;
                    }

                    SystemInfo systemInfo = null;
                    DomainController domainController = null;
                    using (var entityManager = new EntityManager()) {
                        #region Get DomainController
                        domainController = entityManager.Set<DomainController>().Find(dcComputerAccount.ObjectGuid);
                        if (domainController == null) {
                            domainController = entityManager.Set<DomainController>().Create();

                            domainController.Id = dcComputerAccount.ObjectGuid;
                            domainController.ADSite = adSite.Name;
                            domainController.DN = dcComputerAccount.DN;
                            domainController.DNSDomain = dcComputerAccount.DNSDomain;
                            domainController.DNSName = dcComputerAccount.DnsHostName;
                            domainController.NetbiosDomain = dcComputerAccount.Domain;
                            domainController.OSName = dcComputerAccount.OperatingSystem;
                            domainController.OSVersion = dcComputerAccount.OperatingSystemVersionFriendly;
                            domainController.SamAccountName = dcComputerAccount.SamAccountName;
                            domainController.WhenCreated = dcComputerAccount.WhenCreated;
                            domainController.DetachedState = DetachedState.Added;
                        }
                        else {
                            // check if we need to update any of the properties, perhaps due to an OS upgrade or service pack, or moved to another OU
                            bool noChange = true;

                            noChange &= domainController.ADSite == adSite.Name;
                            noChange &= domainController.DN == dcComputerAccount.DN;
                            noChange &= domainController.DNSDomain == dcComputerAccount.DNSDomain;
                            noChange &= domainController.DNSName == dcComputerAccount.DnsHostName;
                            noChange &= domainController.NetbiosDomain == dcComputerAccount.Domain;
                            noChange &= domainController.OSName == dcComputerAccount.OperatingSystem;
                            noChange &= domainController.OSVersion == dcComputerAccount.OperatingSystemVersionFriendly;
                            noChange &= domainController.SamAccountName == dcComputerAccount.SamAccountName;

                            if (!noChange) {
                                Log.WarnFormat("[ThreadID: {0}] {1} Updating DomainController object due to one or more properties are now different from the DC computer account.", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());
                                Log.WarnFormat("[ThreadID: {0}] {1} DomainController custom object properies: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), domainController.ToVerboseString());
                                Log.WarnFormat("[ThreadID: {0}] {1} DC computer account properies: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), dcComputerAccount.ToVerboseString());

                                if (domainController.ADSite != adSite.Name) domainController.ADSite = adSite.Name;
                                if (domainController.DN != dcComputerAccount.DN) domainController.DN = dcComputerAccount.DN;
                                if (domainController.DNSDomain != dcComputerAccount.DNSDomain) domainController.DNSDomain = dcComputerAccount.DNSDomain;
                                if (domainController.DNSName != dcComputerAccount.DnsHostName) domainController.DNSName = dcComputerAccount.DnsHostName;
                                if (domainController.NetbiosDomain != dcComputerAccount.Domain) domainController.NetbiosDomain = dcComputerAccount.Domain;
                                if (domainController.OSName != dcComputerAccount.OperatingSystem) domainController.OSName = dcComputerAccount.OperatingSystem;
                                if (domainController.OSVersion != dcComputerAccount.OperatingSystemVersionFriendly) domainController.OSVersion = dcComputerAccount.OperatingSystemVersionFriendly;
                                if (domainController.SamAccountName != dcComputerAccount.SamAccountName) domainController.SamAccountName = dcComputerAccount.SamAccountName;
                                if (domainController.DetachedState != DetachedState.Unchanged) domainController.DetachedState = DetachedState.Unchanged;
                            }

                            entityManager.Detach(domainController);
                        }
                        #endregion

                        systemInfo = entityManager.Set<SystemInfo>().Create();
                        entityManager.Detach(systemInfo);

                    } // using (var entityManager = new EntityManager())

                    systemInfo.CheckDateTimeUTC = HealthCheckStartTime;
                    systemInfo.DomainController = domainController;
                    systemInfo.DetachedState = DetachedState.Added;

                    #region Populate SystemInfo FSMO Role properties
                    if (FSMORoleOwners.ContainsKey(systemInfo.DomainController.DNSDomain)) {
                        systemInfo.IsPDCEmulator = FSMORoleOwners[systemInfo.DomainController.DNSDomain]["PDCEmulatorOwner"].ToLower()
                            .Contains(systemInfo.DomainController.SamAccountNameFriendly.ToLower());

                        systemInfo.IsInfrastructureManager = FSMORoleOwners[systemInfo.DomainController.DNSDomain]["InfrastractureManagerOwner"].ToLower()
                            .Contains(systemInfo.DomainController.SamAccountNameFriendly.ToLower());

                        systemInfo.IsRIDManager = FSMORoleOwners[systemInfo.DomainController.DNSDomain]["RIDManagerOwner"].ToLower()
                            .Contains(systemInfo.DomainController.SamAccountNameFriendly.ToLower());
                    }
                    #endregion

                    #region Check if the site server is the ISTG
                    systemInfo.IsISTG = (siteServer.NTDSSettings.DN == adSite.NTDSSiteSettings.InterSiteTopologyGenerator);
                    #endregion

                    #region Populate the TagObjectValues collection
                    foreach (var tagObject in tagObjects) {
                        if (!ADDataAccess.ADDomainInfo.Any(x => x.DNSDomainName == tagObject.DNSDomain)) continue;

                        TagObjectValue tagObjectValue = new TagObjectValue();
                        tagObjectValue.TagObject = tagObject;
                        //tagObjectValue.SystemInfo = systemInfo;
                        tagObjectValue.DetachedState = DetachedState.Added;

                        systemInfo.TagObjectValues.Add(tagObjectValue);
                        //tagObject.TagObjectValues.Add(tagObjectValue);
                    }
                    #endregion

                    var systemInfoBuilder = new SystemInfoBuilder(systemInfo, pdcGroupPolicySysvolInfo, siteServer);
                    DCsForTest.Add(systemInfoBuilder);

                } //  foreach (var dcComputerAccount in DCComputerAccounts)

                Log.InfoFormat("[ThreadID: {0}] {1} DomainController objects created for testing: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DCsForTest.Count);

                #region Log ServersNotTested
                ServersNotTestedSummary.AppendLine("SiteServers not tested due to no corresponding computer account:");
                ServersNotTestedSummary.AppendLine();
                foreach (var adSiteToCheck in ADSites.Values) {
                    foreach (var siteServerToCheck in adSiteToCheck.SiteServers) {
                        if (!DCComputerAccounts.Any(x => string.Equals(x.DnsHostName, siteServerToCheck.DnsHostName, StringComparison.OrdinalIgnoreCase))) {
                            Log.WarnFormat("[ThreadID: {0}] {1} Not testing SiteServer due to unable to locate a computer account. SiteServer DN: {2}; ServerReference DN: {3}; SiteServer DNS HostName: {4}; Site: {5}",
                                Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), siteServerToCheck.DN,
                                string.IsNullOrWhiteSpace(siteServerToCheck.ServerReference) ? "N/A" : siteServerToCheck.ServerReference,
                                string.IsNullOrWhiteSpace(siteServerToCheck.DnsHostName) ? "N/A" : siteServerToCheck.DnsHostName, adSiteToCheck.Name);

                            ServersNotTestedSummary.AppendLine(siteServerToCheck.DN);
                        }
                    }
                }

                ServersNotTestedSummary.AppendLine();
                #endregion

                for (int index = 0; index < DCsForTest.Count; index++) {
                    #region for test purposes, intentionally leave one DC untested
                    if ((index == 0) && Debugger.IsAttached) continue;
                    #endregion

                    var systemInfoProvider = DCsForTest[index];
                    Amib.Threading.Func<SystemInfo> getSystemInfoMethod = systemInfoProvider.GetSystemInfo;
                    STP.QueueWorkItem(getSystemInfoMethod);
                }

                if (!STP.WaitForIdle(HealthCheckTimeLimit)) {
                    Log.WarnFormat("[ThreadID: {0}] {1} Canceling remaining threadpool items due to the time limit: {2} has expired. STP Active Threads: {3} Waiting Callbacks: {4}",
                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), HealthCheckTimeLimit.ToString(), STP.ActiveThreads, STP.WaitingCallbacks);

                    try {
                        STP.Cancel(true);
                    }
                    catch (InvalidOperationException e) {
                        if (e.Message.Contains("Collection was modified; enumeration operation may not execute")) throw;
                    }
                }

                STP.Shutdown(true, TimeSpan.Zero);

                Log.InfoFormat("[ThreadID: {0}] {1} Finished getting SystemInfo objects.", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

                ProcessResults();
                UpdateDatabase();
                PurgeDatabaseRecords();

                // Update Tag object value
                foreach (var domainInfo in ADDataAccess.ADDomainInfo) {
                    string healthCheckTagDN = string.Format("{0},{1}", DCHealthCheckTagRDN, domainInfo.DistinguishedName);
                    if (!ADDataAccess.UpdateStringAttribute(healthCheckTagDN, "Description", HealthCheckStartTime.ToString("yyyy-MMM-dd HH:mm"))) {
                        Log.WarnFormat("[ThreadID: {0}] {1} Updating tag object returned false: {2}.  Ensure that the tag object has been created.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), healthCheckTagDN);
                    }
                }

                // send summary email
                Utility.CreateEmailNotification(HealthCheckSummary.ToString());
                Utility.ProcessEmailNotifications();
            }
            catch (Exception e) {
                Log.ErrorFormat("[ThreadID: {0}] {1} {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), e.VerboseExceptionString());
            }
            finally {
                string statistics = string.Format("Start time: {0} Time required: {1} Memory used (bytes): {2}, Processor time: {3}",
                    Process.GetCurrentProcess().StartTime.ToString("yyyy-MMM-dd HH:mm:ss"),
                    DateTime.Now.Subtract(Process.GetCurrentProcess().StartTime).HMFriendly().ToString(),
                    string.Format("{0:n0}", Process.GetCurrentProcess().PeakWorkingSet64),
                    Process.GetCurrentProcess().TotalProcessorTime.HMSFriendly().ToString());

                Log.InfoFormat("[ThreadID: {0}] {1} finished. {2}",
                    Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), statistics);
            }
        }

        private static void GetAppSettings() {

            if (ConfigurationManager.AppSettings["HealthCheckTimeLimit"] != null) {
                HealthCheckTimeLimit = new TimeSpan(0, Convert.ToInt32(ConfigurationManager.AppSettings["HealthCheckTimeLimit"]), 0);
            }
            else {
                HealthCheckTimeLimit = new TimeSpan(0, 75, 0);
            }
            if ((HealthCheckTimeLimit.TotalMinutes < 60) || (HealthCheckTimeLimit.TotalMinutes > 240)) {
                HealthCheckTimeLimit = new TimeSpan(0, 75, 0);
            }

            if (ConfigurationManager.AppSettings["DCHealthCheckTagRDN"] != null) {
                DCHealthCheckTagRDN = ConfigurationManager.AppSettings["DCHealthCheckTagRDN"].ToString();
            }
            if (string.IsNullOrWhiteSpace(DCHealthCheckTagRDN)) {
                DCHealthCheckTagRDN = "CN=DCHealthCheckTag,CN=Users";
            }

            if (ConfigurationManager.AppSettings["RequiredProcesses"] != null) {
                string[] requiredProcesses = ConfigurationManager.AppSettings["RequiredProcesses"].ToString().Split(new char[] { '|' });
                foreach (var requiredProcess in requiredProcesses) {
                    string[] processInfo = requiredProcess.Split(new char[] { ';' });
                    if (processInfo.Count() == 2) {
                        if (!RequiredProcesses.ContainsKey(processInfo[0])) {
                            RequiredProcesses.Add(processInfo[0], processInfo[1]);
                        }
                    }
                }
            }

            if (ConfigurationManager.AppSettings["MaxProcessCPUHoursPerDay"] != null) {
                MaxProcessCPUTimePerDay = new TimeSpan(Convert.ToInt32(ConfigurationManager.AppSettings["MaxProcessCPUHoursPerDay"]), 0, 0);
            }
            if (MaxProcessCPUTimePerDay.TotalHours == 0) {
                MaxProcessCPUTimePerDay = new TimeSpan(8, 0, 0);
            }

            if (ConfigurationManager.AppSettings["CDriveFreeSpaceMBWarning"] != null) {
                CDriveFreeSpaceMBWarning = Convert.ToInt32(ConfigurationManager.AppSettings["CDriveFreeSpaceMBWarning"]);
            }
            if (CDriveFreeSpaceMBWarning == 0) {
                CDriveFreeSpaceMBWarning = 500;
            }
            if (ConfigurationManager.AppSettings["CDriveFreeSpaceMBError"] != null) {
                CDriveFreeSpaceMBError = Convert.ToInt32(ConfigurationManager.AppSettings["CDriveFreeSpaceMBError"]);
            }
            if (CDriveFreeSpaceMBError == 0) {
                CDriveFreeSpaceMBError = 100;
            }

            if (ConfigurationManager.AppSettings["MaxUptimeDays"] != null) {
                MaxUptimeDays = Convert.ToInt32(ConfigurationManager.AppSettings["MaxUptimeDays"]);
            }
            if (MaxUptimeDays == 0) {
                MaxUptimeDays = 100;
            }

            if (ConfigurationManager.AppSettings["MaxNtdsDitSizeMBThreshold"] != null) {
                MaxNtdsDitSizeMBThreshold = Convert.ToInt32(ConfigurationManager.AppSettings["MaxNtdsDitSizeMBThreshold"]);
            }
            if (MaxNtdsDitSizeMBThreshold == 0) {
                MaxNtdsDitSizeMBThreshold = 12000;
            }

            if (ConfigurationManager.AppSettings["MaxTotalHandleCountThreshold"] != null) {
                MaxTotalHandleCountThreshold = Convert.ToInt32(ConfigurationManager.AppSettings["MaxTotalHandleCountThreshold"]);
            }
            if (MaxTotalHandleCountThreshold == 0) {
                MaxTotalHandleCountThreshold = 100000;
            }

            if (ConfigurationManager.AppSettings["MaxTotalThreadCountThreshold"] != null) {
                MaxTotalThreadCountThreshold = Convert.ToInt32(ConfigurationManager.AppSettings["MaxTotalThreadCountThreshold"]);
            }
            if (MaxTotalThreadCountThreshold == 0) {
                MaxTotalThreadCountThreshold = 10000;
            }

            if (ConfigurationManager.AppSettings["MaxPendingReplicationsCountThreshold"] != null) {
                MaxPendingReplicationsCountThreshold = Convert.ToInt32(ConfigurationManager.AppSettings["MaxPendingReplicationsCountThreshold"]);
            }
            if (MaxPendingReplicationsCountThreshold == 0) {
                MaxPendingReplicationsCountThreshold = 5000;
            }

            if (ConfigurationManager.AppSettings["DatabasePurgeIntervalDays"] != null) {
                DatabasePurgeIntervalDays = Convert.ToInt32(ConfigurationManager.AppSettings["DatabasePurgeIntervalDays"]);
            }
            if (DatabasePurgeIntervalDays == 0) {
                DatabasePurgeIntervalDays = 180;
            }

            if (ConfigurationManager.AppSettings["PingLossPercentThreshold"] != null) {
                PingLossPercentThreshold = Convert.ToInt32(ConfigurationManager.AppSettings["PingLossPercentThreshold"]);
            }
            if (PingLossPercentThreshold == 0) {
                PingLossPercentThreshold = 20;
            }

            if (ConfigurationManager.AppSettings["HighCpuProcessesToSuspend"] != null) {
                string[] highCpuProcesses = ConfigurationManager.AppSettings["HighCpuProcessesToSuspend"].ToString().Split(new char[] { '|' });
                foreach (var highCpuProcess in highCpuProcesses) {
                    string[] processInfo = highCpuProcess.Split(new char[] { ';' });
                    if (processInfo.Count() == 2) {
                        if (!HighCpuProcessesToSuspend.ContainsKey(processInfo[0])) {
                            HighCpuProcessesToSuspend.Add(processInfo[0], processInfo[1]);
                        }
                    }
                }
            }

            if (ConfigurationManager.AppSettings["PSSuspendPath"] != null) {
                PSSuspendPath = ConfigurationManager.AppSettings["PSSuspendPath"].ToString();
            }
            if (string.IsNullOrWhiteSpace(PSSuspendPath)) {
                PSSuspendPath = @"C:\Util\SysInternals\PSTools\PSSuspend.exe";
            }

            if (ConfigurationManager.AppSettings["HealthCheckUrl"] != null) {
                HealthCheckUrl = ConfigurationManager.AppSettings["HealthCheckUrl"].ToString();
            }
            if (string.IsNullOrWhiteSpace(HealthCheckUrl)) {
                HealthCheckUrl = string.Empty;
            }

            if (ConfigurationManager.AppSettings["DCNameFilter"] != null) {
                DCNameFilter = ConfigurationManager.AppSettings["DCNameFilter"].ToString();
            }
            if (string.IsNullOrWhiteSpace(DCNameFilter)) {
                DCNameFilter = string.Empty;
            }

            #region Minimum age of computer/NTDS objects.
            if (ConfigurationManager.AppSettings["ServerAgeMinimumThresholdDays"] != null) {
                var days = 0;
                if (int.TryParse(ConfigurationManager.AppSettings["ServerAgeMinimumThresholdDays"].ToString(), out days)) {
                    ServerAgeMinimumThreshold = TimeSpan.FromDays(days);
                }
            }

            if (ConfigurationManager.AppSettings["SiteServerAgeMinimumThresholdDays"] != null) {
                var days = 0;
                if (int.TryParse(ConfigurationManager.AppSettings["SiteServerAgeMinimumThresholdDays"].ToString(), out days)) {
                    SiteServerAgeMinimumThreshold = TimeSpan.FromDays(days);
                }
            }

            if (ConfigurationManager.AppSettings["SiteServerNTDSSettingsAgeMinimumThresholdDays"] != null) {
                var days = 0;
                if (int.TryParse(ConfigurationManager.AppSettings["SiteServerNTDSSettingsAgeMinimumThresholdDays"].ToString(), out days)) {
                    SiteServerNTDSSettingsAgeMinimumThreshold = TimeSpan.FromDays(days);
                }
            } 
            #endregion
        }

        private static void ProcessResults() {
            Log.InfoFormat("[ThreadID: {0}] {1}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

            var startTime = DateTime.UtcNow;

            try {
                // remove servers that may not have been tested
                DCsForTest = DCsForTest.Where(x => x.TestStarted).ToList();

                #region IP Address check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.IPAddress == null) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Error: Unable to get IP Address.");
                    }
                }
                #endregion

                #region PingTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.PingTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Error: Ping test failed. IP Address obtained from DNS: {0}",
                            systemInfo.IPAddress == null ? "N/A" : systemInfo.IPAddress));

                        if (systemInfo.PingLossPct != -1) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  - Ping percent packet loss: {0}", systemInfo.PingLossPct));
                        }
                    }
                    else if (systemInfo.PingLossPct > PingLossPercentThreshold) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Ping packet loss percentage: {0}",
                            systemInfo.PingLossPct));
                    }
                }
                #endregion

                #region DNSTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.DNSTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Error: DNS/Port 53 test failed.");
                    }
                }
                #endregion

                #region LDAPBindTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.LdapBindTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Error: LDAP/Port 389 test failed.");
                    }
                }
                #endregion

                #region GCBindTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.GCBindTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Warning: GC/Port 3268 test failed.");
                    }
                }
                #endregion

                #region SysvolTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.SysvolTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Error: Sysvol test failed.");
                    }
                }
                #endregion

                #region StrictReplicationTest check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.StrictReplicationTest == TestResult.Fail) {
                        systemInfo.ErrorMessageBuilder.AppendLine(" -Error: StrictReplication test failed.");
                        systemInfo.ErrorMessageBuilder.AppendLine(@"  -> Action required: Ensure that the following registry value is set to 1: HKLM\SYSTEM\CurrentControlSet\Services\NTDS\Parameters\Strict Replication Consistency");
                    }
                }
                #endregion

                #region Calculate what the tag objects value should be, using the most recent value
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    foreach (var tagValue in systemInfo.TagObjectValues) {
                        if (!CurrentTagObjectInfo.ContainsKey(tagValue.TagObject.DN)) {
                            CurrentTagObjectInfo.Add(tagValue.TagObject.DN, tagValue.TagValue);
                        }
                        else if (CurrentTagObjectInfo[tagValue.TagObject.DN] < tagValue.TagValue) {
                            CurrentTagObjectInfo[tagValue.TagObject.DN] = tagValue.TagValue;
                        }
                    }
                }

                Log.InfoFormat("[ThreadID: {0}] {1} Current tag values:", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

                foreach (var tag in CurrentTagObjectInfo) {
                    Log.InfoFormat("[ThreadID: {0}] {1} Tag: {2} Value: {3}",
                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), tag.Key, tag.Value.ToString("yyyy-MMM-dd HH:mm"));
                }
                #endregion

                #region Log warnings for tag objects that exceed threshold
                int healthCheckTagMaxDaysThreshold = -2;
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;

                    // skip this check if the server was recently installed
                    if (DateTime.UtcNow.Subtract(systemInfo.OSInstallDate).Days < 2) continue;

                    foreach (var tag in systemInfo.TagObjectValues) {
                        if (tag.TagValue == DateTime.MinValue) continue;

                        if (CurrentTagObjectInfo.ContainsKey(tag.TagObject.DN)) {
                            if (CurrentTagObjectInfo[tag.TagObject.DN].AddDays(healthCheckTagMaxDaysThreshold) > tag.TagValue) {
                                string message = string.Format(" -Warning: Health check tag: {0} value: {1} is older than expected value: {2}",
                                    tag.TagObject.DN, tag.TagValue.ToString("yyyy-MMM-dd HH:mm"), CurrentTagObjectInfo[tag.TagObject.DN].ToString("yyyy-MMM-dd HH:mm"));

                                systemInfo.ErrorMessageBuilder.AppendLine(string.Format("{0}", message));
                                systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Investigate issues that may delay replication (example: disk corruption, lingering objects).");

                                Log.WarnFormat("[ThreadID: {0}] {1} Server: {2} {3}",
                                    Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfo.DomainController.DNSName, message);
                            }
                        }
                    } // foreach (var tag in systemInfo.TagObjectValues)
                } // for (int index = 0; index < DCsForTesting.Count; index++) 
                #endregion

                #region Sysvol replication check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    // we will not have anything to compare if the Sysvol test is not Success
                    if (systemInfo.SysvolTest != TestResult.Success) continue;

                    // the reference GPO is taken from the PDC Emulator, therefore no need to check it
                    if (systemInfo.IsPDCEmulator) continue;

                    if (!DomainPDCSysvolPolicyFolders.ContainsKey(systemInfo.DomainController.DNSDomain)) continue;

                    var pdcGroupPolicySysvolInfo = DomainPDCSysvolPolicyFolders[systemInfo.DomainController.DNSDomain];
                    if (pdcGroupPolicySysvolInfo == null) continue;

                    string sysvolReplicationActionMessage = "  -> Action required: Investigate File Replication Service (FRS) replication issues.  'ntfrsutl' may be used to force replication from PDC emulator, restart FRS service, or in the case of journal wrap, reinitialized with a nonauthoritative restore (BURFLAGS=D2).";
                    if (systemInfo.SysvolReplicationType == SysvolReplicationType.DFSR) {
                        sysvolReplicationActionMessage = "  -> Action required: Investigate DFSR replication issues.  'Dfsrdiag SyncNow ' may be used to force replication; restart DFSR service.";
                    }

                    if (systemInfo.GroupPolicySysvolInfo == null) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Group Policy folder has not replicated to this server:"));
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  -Folder Name: {0} Gpo Display Name: {1} Gpt.Ini Modified (UTC): {2}",
                            pdcGroupPolicySysvolInfo.DirectoryName, pdcGroupPolicySysvolInfo.DisplayName, pdcGroupPolicySysvolInfo.ModifiedUTC.ToString("yyyy-MMM-dd HH:mm")));

                        systemInfo.ErrorMessageBuilder.AppendLine(sysvolReplicationActionMessage);
                    }
                    else if (string.IsNullOrWhiteSpace(systemInfo.GroupPolicySysvolInfo.Version)) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Group Policy folder may be missing one or more files (gpt.ini and/or pol registry.pol)"));
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  -Folder Name: {0} Gpo Display Name: {1}",
                            pdcGroupPolicySysvolInfo.DirectoryName, pdcGroupPolicySysvolInfo.DisplayName));

                        systemInfo.ErrorMessageBuilder.AppendLine(sysvolReplicationActionMessage);
                    }
                    else if (systemInfo.GroupPolicySysvolInfo.Version != pdcGroupPolicySysvolInfo.Version) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Gpt.ini in folder Name: {0} Gpo Display Name: {1} Version: {2}",
                            systemInfo.GroupPolicySysvolInfo.DirectoryName, pdcGroupPolicySysvolInfo.DisplayName, systemInfo.GroupPolicySysvolInfo.Version));
                        systemInfo.ErrorMessageBuilder.AppendLine("  -does not match source gpt.ini:");
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  -Version: {0} Folder: {1}", pdcGroupPolicySysvolInfo.Version, pdcGroupPolicySysvolInfo.DirectoryPath));

                        systemInfo.ErrorMessageBuilder.AppendLine(sysvolReplicationActionMessage);
                    }
                }
                #endregion

                #region Process threshold checks
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.ProcessInfos.Count > 0) {
                        TimeSpan highestProcessorTimePerDay = new TimeSpan(0, 0, 0);
                        TimeSpan highestProcessorTime = new TimeSpan(0, 0, 0);
                        string highestProcessorTimePerDayProcess = string.Empty;
                        string highestProcessorTimeProcess = string.Empty;

                        foreach (var process in systemInfo.ProcessInfos) {
                            if (process.ProcessId < 5) continue;
                            if (process.ExecutablePath.EndsWith("lsass.exe")) continue;
                            if (process.ExecutablePath.EndsWith("dns.exe")) continue;
                            if (systemInfo.UptimeDays < 1) continue;

                            // try to correlate the process with a specific service.  
                            // If the process is svchost, it may host multiple services, so that may not be possible
                            // If the process has been orphaned, the service may not be found.  I.e., the service was restarted,
                            // has a new service process, and the old process is still running. (multiple instances)
                            ServiceInfo service = null;
                            if ((process.ProcessId > 0) && !process.ExecutablePath.EndsWith("svchost.exe", StringComparison.OrdinalIgnoreCase)) {
                                var services = systemInfo.ServiceInfos.Where(x =>
                                    (x.ProcessId == process.ProcessId) &&
                                    x.PathName.ToLower().Contains(process.ExecutablePath.ToLower()))
                                .ToList();

                                if (services.Count == 1) {
                                    service = services.Single();
                                }
                            }

                            #region Log the process with highest Processor Time
                            if (process.ProcessorTime > highestProcessorTime) {
                                highestProcessorTime = process.ProcessorTime;
                                highestProcessorTimeProcess = process.ExecutablePath;
                            }
                            if (process.ProcessorTimePerDay > highestProcessorTimePerDay) {
                                highestProcessorTimePerDay = process.ProcessorTimePerDay;
                                highestProcessorTimePerDayProcess = process.ExecutablePath;
                            }
                            #endregion

                            if ((process.ProcessorTime > MaxProcessCPUTimePerDay) && (process.ProcessorTimePerDay > MaxProcessCPUTimePerDay)) {
                                if (File.Exists(PSSuspendPath) && HighCpuProcessesToSuspend.Any(x => process.ExecutablePath.ToLower().Contains(x.Key.ToLower()))) {
                                    Log.WarnFormat("[ThreadID: {0}] {1} Server: {2} Suspending known offending high cpu process: {3} ProcessorTimePerDay: {4}",
                                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfo.DomainController.DNSName, process.ExecutablePath, process.ProcessorTimePerDay.ToString());

                                    string commandArguments = string.Format(@" /accepteula \\{0} {1}", systemInfo.DomainController.DNSName, process.ProcessId);
                                    List<string> results = Utility.ExecuteShellCommand(PSSuspendPath, commandArguments);

                                    Log.WarnFormat("[ThreadID: {0}] {1} Server: {2} Suspending known offending high cpu process: {3} Results:",
                                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfo.DomainController.DNSName, process.ExecutablePath);

                                    Log.WarnFormat("[ThreadID: {0}] {1} {2}",
                                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), results.ToFormattedString());
                                }
                                else {
                                    systemInfo.ErrorMessageBuilder.AppendLine(" -Warning: High Average Daily Process CPU Utilization:");
                                    systemInfo.ErrorMessageBuilder.AppendLine(string.Format("   -Process Id: {0} CPU time per day: {1} Total CPU time: {2} Elapsed time: {3}",
                                        process.ProcessId.ToString(), process.ProcessorTimePerDay.ToString(), process.ProcessorTime.ToString(), process.ElapsedTime.ToString()));
                                    systemInfo.ErrorMessageBuilder.AppendLine(string.Format("   -Path: {0}", process.ExecutablePath));
                                    systemInfo.ErrorMessageBuilder.AppendLine(string.Format("   -Command Line: {0}", process.CommandLine));
                                    if (service != null) {
                                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("   -Service: Name: {0} DisplayName: {1}", service.Name, service.DisplayName));
                                    }

                                    systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Investigate as possible runaway process.  The process/service may require terminating/restarting");
                                }
                            }
                        } // foreach (var process in systemInfo.ProcessInfos)
                        /*
                        Log.InfoFormat("[ThreadID: {0}] {1} Highest ProcessorTimePerDay process: {2} Time: {3}", 
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), highestProcessorTimePerDayProcess, highestProcessorTimePerDay.ToString());
                        Log.InfoFormat("[ThreadID: {0}] {1} Highest ProcessorTime process: {2} Time: {3}",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), highestProcessorTimeProcess, highestProcessorTime.ToString());
                        */
                        foreach (var requiredProcess in RequiredProcesses) {
                            if (!systemInfo.ProcessInfos.Any(x => x.ExecutablePath.Replace("\"", string.Empty).EndsWith(requiredProcess.Key, StringComparison.OrdinalIgnoreCase))) {
                                systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Process not found: {0} ({1})",
                                    requiredProcess.Key, requiredProcess.Value));

                                systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Restart the service, or install the application if it is not present.");
                            }
                        }
                    } // if (this.Processes.Count > 0)
                } // for (int index = 0; index < DCsForTesting.Count; index++) 
                #endregion

                #region Disk space check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.CDriveFreeMB > -1) {
                        if (systemInfo.CDriveFreeMB < CDriveFreeSpaceMBError) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Error: C Drive free space is: {0} MBytes.", systemInfo.CDriveFreeMB.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Investigate cause of disk space depletion; free up disk space.");
                        }
                        else if (systemInfo.CDriveFreeMB < CDriveFreeSpaceMBWarning) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: C Drive free space is: {0} MBytes.", systemInfo.CDriveFreeMB.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Investigate cause of disk space depletion; free up disk space.");
                        }
                    }
                }
                #endregion

                #region NTDS.dit size check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.NTDSDitSizeMB > -1) {
                        if (systemInfo.NTDSDitSizeMB > MaxNtdsDitSizeMBThreshold) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Ntds.dit size is: {0} Mbytes.", systemInfo.NTDSDitSizeMB.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: System Engineering: Investigate cause of database size increase.");
                        }
                    }
                }
                #endregion

                #region Handle count check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.TotalHandles > -1) {
                        if (systemInfo.TotalHandles > MaxTotalHandleCountThreshold) {
                            var highHandleCountProcess = systemInfo.ProcessInfos.OrderByDescending(x => x.HandleCount).FirstOrDefault();

                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Total Handle count is: {0}.  Process with highest handle count: {1}.  Handle count: {2}",
                                systemInfo.TotalHandles.ToString(), highHandleCountProcess != null ? highHandleCountProcess.ExecutablePath : "N/A", highHandleCountProcess.HandleCount.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: System Engineering: Investigate possible handle leak in a process");
                        }
                    }
                }
                #endregion

                #region Thread count check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.TotalThreads > -1) {
                        if (systemInfo.TotalThreads > MaxTotalThreadCountThreshold) {
                            var highThreadCountProcess = systemInfo.ProcessInfos.OrderByDescending(x => x.ThreadCount).FirstOrDefault();

                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Total Thread count is: {0}.  Process with highest thread count: {1}.  Thread count: {2}",
                                systemInfo.TotalHandles.ToString(), highThreadCountProcess != null ? highThreadCountProcess.ExecutablePath : "N/A", highThreadCountProcess.ThreadCount.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: System Engineering: Investigate possible resource leak in a process");
                        }
                    }
                }
                #endregion

                #region Pending Replications count check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.PendingReplications.Count > -1) {
                        // skip this check if the server was recently installed
                        if (DateTime.UtcNow.Subtract(systemInfo.OSInstallDate).Days < 2) continue;

                        if (systemInfo.PendingReplications.Count > MaxPendingReplicationsCountThreshold) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Pending Replications count is: {0}.",
                                systemInfo.PendingReplications.Count.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: System Engineering: Investigate possible cause for replication delays.");
                        }
                    }
                }
                #endregion

                #region Uptime check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.UptimeDays > -1) {
                        if (systemInfo.UptimeDays > MaxUptimeDays) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: System uptime is: {0} days.", systemInfo.UptimeDays.ToString()));
                            systemInfo.ErrorMessageBuilder.AppendLine("  -> Action required: Investigate if previous restarts have failed or hotfixes are not being applied.");
                        }
                    }
                }
                #endregion

                #region DNS PublishAddresses registry setting check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (string.IsNullOrWhiteSpace(systemInfo.DomainController.OSName) || systemInfo.DomainController.OSName.Contains("2003")) continue;
                    if (string.IsNullOrWhiteSpace(systemInfo.IPAddresses)) continue;
                    if (!systemInfo.IPAddresses.Contains(",")) continue;

                    string[] ipAddresses = systemInfo.IPAddresses.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                    if (ipAddresses.Length > 1) {
                        if (string.IsNullOrWhiteSpace(systemInfo.DNSPublishAddresses)) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Multiple IP Addresses found: {0}, but unable to confirm that registry value PublishAddresses exists.", systemInfo.IPAddresses));
                            systemInfo.ErrorMessageBuilder.AppendLine(@"  -> Action required: Create REG_SZ registry value: PublishAddresses Under key: HKLM\System\CurrentControlSet\Services\DNS\Parameters for the desired IP address.");
                        }

                        else if (!ipAddresses.Any(x => x == systemInfo.DNSPublishAddresses)) {
                            systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Multiple IP Addresses found: {0}, but unable to confirm that registry value PublishAddresses: {1} matches an existing IP address.", systemInfo.IPAddress, systemInfo.DNSPublishAddresses));
                            systemInfo.ErrorMessageBuilder.AppendLine(@"  -> Action required: Update REG_SZ registry value: PublishAddresses Under key: HKLM\System\CurrentControlSet\Services\DNS\Parameters for the desired IP address.");
                        }
                    }
                }
                #endregion

                #region DNS ScavengingInterval registry setting check for FSMO role owners
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (systemInfo.DNSScavengingInterval < 1) continue;

                    if ((systemInfo.IsPDCEmulator) || (systemInfo.IsInfrastructureManager) || (systemInfo.IsRIDManager)) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Server is a FSMO role owner and is configured for a DNS Scavenging interval of: {0} hours.", systemInfo.DNSScavengingInterval));
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  - Best practice specifies DNS Scavenging should be performed on only one DNS Server, and that server should not be a FSMO role owner."));
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format("  - FSMO roles: PDC Emulator: {0} Infrastructure Manager: {1} RID Manager: {2}",
                            systemInfo.IsPDCEmulator.ToString(), systemInfo.IsInfrastructureManager.ToString(), systemInfo.IsRIDManager.ToString()));
                        systemInfo.ErrorMessageBuilder.AppendLine(@"  -> Action required: Disable DNS Scavenging on this domain controller. DNS Manager > Server properties > Advanced tab > Uncheck box for 'Enable Automatic Scavenging'");
                    }
                }
                #endregion

                #region DNSResponse multiple IP address check
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (string.IsNullOrWhiteSpace(systemInfo.DNSResponse)) continue;
                    if (!systemInfo.DNSResponse.Contains(",")) continue;

                    string[] ipAddresses = systemInfo.DNSResponse.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                    if (ipAddresses.Length > 1) {
                        systemInfo.ErrorMessageBuilder.AppendLine(string.Format(" -Warning: Multiple IP Addresses returned for DNS query for DC host name: {0}", systemInfo.DNSResponse));
                        systemInfo.ErrorMessageBuilder.AppendLine(@"  -> Action required: Confirm that the registry value exists for a single IP Address: PublishAddresses Under key: HKLM\System\CurrentControlSet\Services\DNS\Parameters");
                    }
                }
                #endregion

                #region copy the accumulated formatted error messages from the stringbuilder to the Error propery
                for (int index = 0; index < DCsForTest.Count; index++) {
                    var systemInfo = DCsForTest[index].SystemInfo;
                    if (string.IsNullOrWhiteSpace(systemInfo.ErrorMessage) && (systemInfo.ErrorMessageBuilder.Length > 0)) {
                        systemInfo.ErrorMessage = systemInfo.ErrorMessageBuilder.ToString();
                    }
                }
                #endregion

                #region Log important statistics
                TimeSpan timeRequired = DateTime.Now.Subtract(Process.GetCurrentProcess().StartTime);
                HealthCheckSummary.AppendLine(string.Format("DC Health Check finished at: {0} (UTC). Site count: {1}  Servers tested: {2}  Time required: {3} minutes",
                    DateTime.UtcNow.ToString("yyyy-MMM-dd HH:mm"), ADSites.Count, DCsForTest.Count, (int)timeRequired.TotalMinutes));
                HealthCheckSummary.AppendLine();

                int officeScanCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("ntrtscan.exe", StringComparison.OrdinalIgnoreCase))).Count();
                int deepSecurityCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("ds_agent.exe", StringComparison.OrdinalIgnoreCase))).Count();
                int snareCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("snare.exe", StringComparison.OrdinalIgnoreCase))).Count();
                int scomCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("healthservice.exe", StringComparison.OrdinalIgnoreCase))).Count();
                int bigfixCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("besclient.exe", StringComparison.OrdinalIgnoreCase))).Count();
                int besRelayCount = DCsForTest.Where(x => x.SystemInfo.ProcessInfos.Any(y => y.ExecutablePath.Replace("\"", string.Empty).EndsWith("besrelay.exe", StringComparison.OrdinalIgnoreCase))).Count();

                int officeScanPercent = (int)(((double)officeScanCount / (double)DCsForTest.Count) * 100);
                int deepSecurityPercent = (int)(((double)deepSecurityCount / (double)DCsForTest.Count) * 100);
                int snarePercent = (int)(((double)snareCount / (double)DCsForTest.Count) * 100);
                int scomPercent = (int)(((double)scomCount / (double)DCsForTest.Count) * 100);
                int bigfixPercent = (int)(((double)bigfixCount / (double)DCsForTest.Count) * 100);
                int besrelayPercent = (int)(((double)besRelayCount / (double)DCsForTest.Count) * 100);

                int serversWithIssues = DCsForTest.Where(x => !string.IsNullOrWhiteSpace(x.SystemInfo.ErrorMessage)).Count();

                HealthCheckSummary.AppendLine(string.Format("Trend OfficeScan Realtime Monitor is running on: {0} of {1} Servers. ({2}%)", officeScanCount, DCsForTest.Count, officeScanPercent));
                HealthCheckSummary.AppendLine(string.Format("Trend Deep Security Agent is running on: {0} of {1} Servers. ({2}%)", deepSecurityCount, DCsForTest.Count, deepSecurityPercent));
                HealthCheckSummary.AppendLine(string.Format("Snare Syslog Agent is running on: {0} of {1} Servers. ({2}%)", snareCount, DCsForTest.Count, snarePercent));
                HealthCheckSummary.AppendLine(string.Format("SCOM 2007 is running on: {0} of {1} Servers. ({2}%)", scomCount, DCsForTest.Count, scomPercent));
                HealthCheckSummary.AppendLine(string.Format("BigFix Client is running on: {0} of {1} Servers. ({2}%)", bigfixCount, DCsForTest.Count, bigfixPercent));
                HealthCheckSummary.AppendLine(string.Format("BigFix Relay is running on: {0} of {1} Servers. ({2}%)", besRelayCount, DCsForTest.Count, besrelayPercent));
                HealthCheckSummary.AppendLine();

                if (!string.IsNullOrWhiteSpace(HealthCheckUrl)) {
                    HealthCheckSummary.AppendLine(string.Format("Detailed results available at: {0}", HealthCheckUrl));
                    HealthCheckSummary.AppendLine();
                }

                if (serversWithIssues == 0) {
                    HealthCheckSummary.AppendLine("No issues found.");
                }
                else {
                    HealthCheckSummary.AppendLine(string.Format("{0} Servers with issues:", serversWithIssues));
                    HealthCheckSummary.AppendLine();

                    for (int index = 0; index < DCsForTest.Count; index++) {
                        var systemInfo = DCsForTest[index].SystemInfo;
                        if (string.IsNullOrWhiteSpace(systemInfo.ErrorMessage)) continue;

                        HealthCheckSummary.AppendLine(string.Format("Server: {0} Issues:", systemInfo.DomainController.DNSName));
                        HealthCheckSummary.AppendLine(systemInfo.ErrorMessage.ToString());
                    }

                    HealthCheckSummary.AppendLine();
                }

                #endregion

                Log.InfoFormat("[ThreadID: {0}] {1} Report Summary: {2}",
                    Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), HealthCheckSummary.ToString());

                File.WriteAllText(ReportSummaryPath, HealthCheckSummary.ToString());
            }
            catch (Exception e) {
                Log.ErrorFormat("[ThreadID: {0}] {1} {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), e.VerboseExceptionString());
                throw;
            }
            finally {
                Log.InfoFormat("[ThreadID: {0}] {1} finished.  Time required: {2}.  Sites found: {3}; Servers found in sites: {4}",
                    Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DateTime.UtcNow.Subtract(startTime).ToString(),
                    ADSites.Count, ADSites.Values.Sum(x => x.SiteServers.Count));
            }
        }

        private static void PurgeDatabaseRecords() {
            Log.InfoFormat("[ThreadID: {0}] {1}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

            var startTime = DateTime.UtcNow;

            try {
                using (var entityManager = new EntityManager()) {
                    var purgeDate = DateTime.UtcNow.Date.AddDays(-DatabasePurgeIntervalDays);
                    var systemInfoRecordsToPurge = entityManager.QuerySets.SystemInfosSlim
                        .Where(x => x.CheckDateTimeUTC < purgeDate)
                        .ToList();

                    if (systemInfoRecordsToPurge.Count > 0) {
                        Log.InfoFormat("[ThreadID: {0}] {1} Removing database records older than: {2}.",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), purgeDate.ToString("yyyy-MMM-dd"));
                        Log.InfoFormat("[ThreadID: {0}] {1} Number of old records to delete: {2}",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfoRecordsToPurge.Count);

                        for (int index = 0; index < systemInfoRecordsToPurge.Count; index++) {
                            var systemInfoRecord = systemInfoRecordsToPurge[index];
                            systemInfoRecord.DetachedState = DetachedState.Deleted;
                            entityManager.Set<SystemInfo>().Remove(systemInfoRecord);
                        }
                    }

                    var filter = TestResult.NotPerformed;
                    var testNotPerformedRecordsToPurge = entityManager.QuerySets.SystemInfosSlim
                        .Where(x => x.PingTest == filter)
                        .ToList();

                    for (int index = 0; index < testNotPerformedRecordsToPurge.Count; index++) {
                        var systemInfoRecord = systemInfoRecordsToPurge[index];
                        systemInfoRecord.DetachedState = DetachedState.Deleted;
                        entityManager.Set<SystemInfo>().Remove(systemInfoRecord);
                    }

                    entityManager.SaveChanges();
                }
            }
            catch (Exception e) {
                Log.ErrorFormat("[ThreadID: {0}] {1} {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), e.VerboseExceptionString());
                throw;
            }
            finally {
                Log.InfoFormat("[ThreadID: {0}] {1} finished.  Time required: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DateTime.UtcNow.Subtract(startTime).ToString());
            }
        }

        private static void STPPostExecute(IWorkItemResult wir) {
            if (STP.IsShuttingdown) return;
            Log.DebugFormat("[ThreadID: {0}] {1} STP Active Threads: {2} Waiting Callbacks: {3}",
                Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), STP.ActiveThreads, STP.WaitingCallbacks);

            if (wir.Exception != null) {
                var exception = wir.Exception as Exception;

                if (exception != null) {
                    Log.WarnFormat("[ThreadID: {0}] {1} STP Exception: {2}",
                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), exception.VerboseExceptionString());
                }
            }

            // if we are near the completion of remaining tasks in the threadpool, record the remaining tasks.
            // this can help identify which threads are hung or are blocking completion
            if ((STP.WaitingCallbacks == 0) && (STP.ActiveThreads > 0) && (STP.ActiveThreads < 10)) {
                var remainingTasks = new List<SystemInfoBuilder>();
                lock (DCsForTest) {
                    for (int index = 0; index < DCsForTest.Count; index++) {
                        if (!DCsForTest[index].TestCompleted) {
                            remainingTasks.Add(DCsForTest[index]);
                        }
                    }
                }

                if (remainingTasks.Count > 0) {
                    Log.DebugFormat("[ThreadID: {0}] {1} - STP Remaining Tasks:", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

                    for (int index = 0; index < remainingTasks.Count; index++) {
                        var systemInfoProvider = remainingTasks[index];

                        // if SystemInfo were detached, DomainController will be null
                        if (systemInfoProvider.SystemInfo == null) continue;
                        if (systemInfoProvider.SystemInfo.DomainController == null) continue;

                        Log.DebugFormat("[ThreadID: {0}] {1} - STP Remaining Task DNSName: {2} VerboseLog Info:{3}{4}",
                            Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfoProvider.SystemInfo.DomainController.DNSName,
                            Environment.NewLine, systemInfoProvider.VerboseLog.ToString());
                    }

                    Log.DebugFormat("[ThreadID: {0}] {1} STP Active Threads: {2} Waiting Callbacks: {3}",
                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), STP.ActiveThreads, STP.WaitingCallbacks);
                }
            }
        }

        private static void TestDatabaseAccess() {
            Log.InfoFormat("[ThreadID: {0}] {1}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

            var startTime = DateTime.UtcNow;

            try {
                using (var entityManager = new EntityManager()) {
                    int recordCount = entityManager.QuerySets.SystemInfosSlimNoTracking.Count();
                    var mostRecentSystemInfoRecord = entityManager.QuerySets.SystemInfosSlimNoTracking
                        .OrderByDescending(x => x.CheckDateTimeUTC)
                        .FirstOrDefault();

                    Log.InfoFormat("[ThreadID: {0}] {1} Database record count: {2}; Most recent database record: {3}",
                        Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(),
                        recordCount,
                        mostRecentSystemInfoRecord != null ? mostRecentSystemInfoRecord.CheckDateTimeUTC.ToString("yyyy-MMM-dd HH:mm") : "N/A");
                }
            }
            catch (Exception e) {
                Log.FatalFormat("[ThreadID: {0}] {1} {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), e.VerboseExceptionString());
                throw;
            }
            finally {
                Log.InfoFormat("[ThreadID: {0}] {1} finished.  Time required: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DateTime.UtcNow.Subtract(startTime).ToString());
            }
        }

        private static void UpdateDatabase() {
            Log.InfoFormat("[ThreadID: {0}] {1}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName());

            var startTime = DateTime.UtcNow;

            try {
                using (var entityManager = new EntityManager()) {
                    var healthCheckSummary = entityManager.Set<HealthCheckSummary>().Create();
                    healthCheckSummary.DetachedState = DetachedState.Added;
                    healthCheckSummary.CheckDateTimeUTC = HealthCheckStartTime;
                    healthCheckSummary.DCsTested = DCsForTest.Count;
                    healthCheckSummary.DCsWithErrors = DCsForTest.Where(x => !string.IsNullOrWhiteSpace(x.SystemInfo.ErrorMessage)).Count();
                    healthCheckSummary.Summary = HealthCheckSummary.ToString();

                    var systemInfosToDetach = new List<Guid>();
                    for (int index = 0; index < DCsForTest.Count; index++) {
                        var systemInfo = DCsForTest[index].SystemInfo;
                        if ((!DCsForTest[index].TestStarted) || (!DCsForTest[index].TestCompleted)) {
                            systemInfosToDetach.Add(systemInfo.Id);
                            continue;
                        }

                        var validationResults = systemInfo.Validate(null).ToList();
                        if (validationResults.Count > 0) {
                            Log.WarnFormat("[ThreadID: {0}] {1} DC: {2} SystemInfo failed validation:",
                                Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), systemInfo.DomainController.DNSName);

                            foreach (var result in validationResults) {
                                Log.WarnFormat("[ThreadID: {0}] {1} Property(s): {2} Validation Error Message: {3}",
                                    Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(),
                                    result.MemberNames.ToList().ToDelimitedString(), result.ErrorMessage);
                            }

                            systemInfosToDetach.Add(systemInfo.Id);
                            continue;
                        }

                        entityManager.Set<SystemInfo>().Add(systemInfo);
                        healthCheckSummary.SystemInfos.Add(systemInfo);

                        var tagObjectValues = systemInfo.TagObjectValues.ToList();
                        for (int tagObjectValueIndex = 0; tagObjectValueIndex < tagObjectValues.Count; tagObjectValueIndex++) {
                            var tag = tagObjectValues[tagObjectValueIndex];
                            if (tag.TagValue == DateTime.MinValue) {
                                if (systemInfo.TagObjectValues.Contains(tag)) systemInfo.TagObjectValues.Remove(tag);
                                if (tag.TagObject.TagObjectValues.Contains(tag)) tag.TagObject.TagObjectValues.Remove(tag);
                                entityManager.Detach(tag);
                            }
                        }
                    } // for (int index = 0; index < DCsForTest.Count; index++)

                    for (int index = 0; index < systemInfosToDetach.Count; index++) {
                        var systemInfo = DCsForTest.Where(x => x.SystemInfo.Id == systemInfosToDetach[index]).First().SystemInfo;
                        systemInfo.DetachedState = DetachedState.Unchanged;
                        entityManager.Detach(systemInfo);
                    }

                    try {
                        entityManager.SaveChanges();
                    }
                    catch (DbUpdateConcurrencyException) {
                        foreach (var entry in entityManager.ChangeTracker.Entries().Where(x => x.State == EntityState.Modified).ToList()) {
                            entityManager.ObjectContext.Refresh(System.Data.Objects.RefreshMode.ClientWins, entry.Entity);
                        }
                    }
                }
            }
            catch (Exception e) {
                Log.ErrorFormat("[ThreadID: {0}] {1} {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), e.VerboseExceptionString());
                throw;
            }
            finally {
                Log.InfoFormat("[ThreadID: {0}] {1} finished.  Time required: {2}", Thread.CurrentThread.ManagedThreadId, Utility.CurrentMethodName(), DateTime.UtcNow.Subtract(startTime).ToString());
            }
        } 
        #endregion
    }
}
